import java.sql.*;

import javax.sql.*;
import java.io.FileWriter;
import java.io.BufferedWriter;

import java.io.IOException;

import java.util.regex.*;

import java.lang.*;
import java.util.*;
import java.util.Date;

import org.apache.commons.lang.StringEscapeUtils;


public class GetSuffix
{
	private static HashSet<String> srcSuffixSet = null;
	private static HashSet<String> buildSuffixSet = null;
	private static HashSet<String> docSuffexSet = null;
	
	//the file to store extracted filepath suffix
	String dstFile = null;
	
	//db object
	DB db = null;
	
	//the number of rows fetched each time
	public static final int ROW_NUM_LIMIT = 1000;
	public static final int INSERT_BATCH_SIZE = 500;
	
	public static final int SRC = 1;
	public static final int BUILD = 2;
	public static final int DOC = 3;
	public static final int TEST = 4;
	
	private final int plotStartYear = 2009;
	private final int plotStartMonth = 1;
	
	Date plotStartDate = null;
	
	private final int plotEndtYear = 2011;
	private final int plotEndMonth = 10;
	
	Date plotEndDate = null;
	
	private HashMap<Integer, Integer> srcDistributionMap = null;
	private HashMap<Integer, Integer> buildDistributionMap = null;
	private HashMap<Integer, Integer> testDistributionMap = null;
	private HashMap<Integer, Integer> docDistributionMap = null;
	
	int batchSize = 0;
	
	//insert file name into table 
	int insertCount = 0;
	PreparedStatement inserPs = null;
	
	//matcher object
	Matcher buildMatch = null;
	Matcher srcMatch = null;
	Matcher docMatch = null;
	Matcher testMatch = null;
	
	Matcher buildPathMatch = null;
	Matcher srcPathMatch = null;
	Matcher docPathMatch = null;
	Matcher testPathMatch = null;
	
	public GetSuffix(int batchSize) throws Exception
	{
		//get the filepath
		this.batchSize = batchSize;
		
		//connect to the db
		db = new DB();
		db.connect();
		
		//to store suffix
		//suffix = new HashSet<String>();
		
		//build this table
		//db.dropTable("filename");
		//db.buildTable("filename");
		
		inserPs = db.getPreparedStatement("filename");
		

		
		//src
		String srcRegex = "(diff)|(jar)|(asm)|(java)|(class)|(source)|(^c$)|(^h$)|(^cpp$)"
				+"|(^py$)|(^el$)|(^elc$)|(^c$)|(^a$)|(^sed$)";
		srcRegex += "|(^awk$)|(^asm$)|(script)|(^sql$)|(python)|(^so$)|(^sh$)|(^php$)";
		
		Pattern srcPattern = Pattern.compile(srcRegex, Pattern.CASE_INSENSITIVE);	
		srcMatch = srcPattern.matcher("");
		
		String srcPathRegex = "(^src$)|(^source$)";
		
		Pattern srcPathpattern = Pattern.compile(srcPathRegex, Pattern.CASE_INSENSITIVE);
		srcPathMatch = srcPathpattern.matcher("");
		
		
		
		
		//build
		String buildRegex = "((configure)|(config)|(conf)|(make)|(build)|(bash)|(^m4$))|(^\\d+$)";
		buildRegex += "|(crt)|(crl)|(^m&)|(^ml$)|(^o$)|(^a$)|(^exe$)|(386)|(x64)|(^rpm$)|(^out$)";
		buildRegex += "|(^run$)";
		
		Pattern buildPattern = Pattern.compile(buildRegex, Pattern.CASE_INSENSITIVE);	
		buildMatch = buildPattern.matcher("");
	
		String buildPathRegex = "(^obj$)|(^bin$)|(^object$)|(^binary$)";
		
		Pattern buildPathPattern = Pattern.compile(buildRegex, 
				Pattern.CASE_INSENSITIVE);	
		buildPathMatch = buildPathPattern.matcher("");
		
		
		
		
		//doc
		String docRegex = "(doc)|(document)|(subversion)|(doc)|(html)|(docx)|(txt)|"
		+"(text)|(readme)" 
		+ "|(install)|(^dot$)|(license)|(log)";	
		
		Pattern docPattern = Pattern.compile(docRegex, Pattern.CASE_INSENSITIVE);	
		docMatch = docPattern.matcher("^obj$");
		
		String docPathRegex = "(^doc$)|(^document$)";
			
		Pattern docPathPattern = Pattern.compile(docRegex, Pattern.CASE_INSENSITIVE);	
		docPathMatch = docPathPattern.matcher("");
		
		
		
		//test
		String testRegex = "(test)";
		
		Pattern testPattern = Pattern.compile(testRegex, Pattern.CASE_INSENSITIVE);	
		testMatch = testPattern.matcher("");
		
		String testPathRegex = ".test.";

		Pattern testPathPattern = Pattern.compile(testRegex, Pattern.CASE_INSENSITIVE);	
		testPathMatch = testPathPattern.matcher("");
		
		srcDistributionMap = new HashMap<Integer, Integer>();
		buildDistributionMap = new HashMap<Integer, Integer>();
		testDistributionMap = new HashMap<Integer, Integer>();
		docDistributionMap = new HashMap<Integer, Integer>();
		

		
		SimpleTimeZone tZone = new SimpleTimeZone(0, "");
		Calendar dayTime = Calendar.getInstance();
		
		dayTime.setTimeZone(tZone);
		
		//set up the time
		dayTime.set(plotStartYear,  plotStartMonth, 1, 1, 1, 1);
		
		plotStartDate = new Timestamp(dayTime.getTimeInMillis());
		
		dayTime.set(plotEndtYear,  plotEndMonth, 30, 23, 59, 59);
		
		plotEndDate = new Timestamp(dayTime.getTimeInMillis());
		
		//System.out.println(plotStartDate.toString());
		
		//System.out.println(plotEndDate.toString());
		
	}
	
	
	
	private int toSuffix(String filepath)
	{
		//break it down into consecutive tokens
		String[] words = filepath.split("\\W");
		
		String suffix = words[words.length - 1].toLowerCase();
		
		for (int i = 0; i < words.length - 2; i++)
		{
			//System.out.println(words[i]);
			
			if (testPathMatch.reset(words[i]).find())
			{
				return TEST;
			}
			
			if (buildPathMatch.reset(words[i]).find())
			{
				return BUILD;
			}
			
			if (srcPathMatch.reset(words[i]).find())
			{
				return SRC;
			}
			
			if (docPathMatch.reset(words[i]).find())
			{
				return DOC;
			}		
		}
		
		return -1;
		
	}
	
	private void insertTable(String filename) throws SQLException
	{
		if (filename != null && filename.length() > 3)
		{
			inserPs.setString(1, filename);
			
			//add to batch
			inserPs.addBatch();
			
			insertCount++;
			
			if (insertCount >= INSERT_BATCH_SIZE)
			{
				insertCount = 0;
				
				inserPs.executeBatch();
			}
		}
	}
	
	//select from db, with the give limit of how many rows that are fetched each time
	private void select(int rowNumLimit) throws SQLException, IOException
	{
		String sqlStatement = new String("SELECT * FROM file");
		PreparedStatement ps = null;
		
		try
		{
			ps = db.getStreamPreparedStatement(sqlStatement, rowNumLimit);
			
		}
		catch (Exception e)
		{
			e.printStackTrace();
		}
		
		ResultSet rs = ps.executeQuery();
		
		int count = 0;
		int suffixCut = 0;
		while (rs.next())
		{
			String filename = rs.getString(1);
			
			int category = classify(rs.getTimestamp(2));
			
			if (category > -1)
			{
			
				filename = StringEscapeUtils.unescapeHtml(filename);
				
				int pattern = toSuffix(filename);
				switch(pattern)
				{
					case BUILD:
					{
						addMap(category, buildDistributionMap);
						
						
						
						break;
					}
					case SRC:
					{
						addMap(category, srcDistributionMap);
						
						break;
					}
					case TEST:
					{
						addMap(category, testDistributionMap);
						
						break;
					}
					case DOC:
					{
						addMap(category, docDistributionMap);
						
						break;
					}
					default:
					{
						suffixCut--;
					}
				}
			}
			count++;
			suffixCut++;
		}
		
		//inserPs.executeBatch();
		
		System.out.println("actual fetched rows: --> " + count);
		System.out.println("actual cut files: --> " + suffixCut);
		
		
		
		
		Set<Integer> categoryList = null;
		
		
		//write to src file
		
		System.out.println(srcDistributionMap.size() + " different src distributions");
		
		FileWriter fstream = new FileWriter("data/srcDistribution.txt");
		
		BufferedWriter out = new BufferedWriter(fstream);
	
		out.write("#" + srcDistributionMap.size());
		
		//iteratively push the suffices into file
		categoryList = srcDistributionMap.keySet();
		for (Integer category: categoryList)
		{
			int num = srcDistributionMap.get(category);
			
			out.write(category + " " + num + "\n");
		}
		
		out.close();
		

		//write to test file
		
		System.out.println(srcDistributionMap.size() + " different test distributions");
		
		fstream = new FileWriter("data/testDistribution.txt");
		
		out = new BufferedWriter(fstream);
	
		out.write("#" + testDistributionMap.size());
		
		//iteratively push the suffices into file
		categoryList = testDistributionMap.keySet();
		for (Integer category: categoryList)
		{
			int num = testDistributionMap.get(category);
			
			out.write(category + " " + num + "\n");
		}
		
		out.close();

		
		//write to doc file
		
		System.out.println(docDistributionMap.size() + " different doc distributions");
		
		fstream = new FileWriter("data/docDistribution.txt");
		
		out = new BufferedWriter(fstream);
	
		out.write("#" + docDistributionMap.size());
		
		//iteratively push the suffices into file
		categoryList = docDistributionMap.keySet();
		for (Integer category: categoryList)
		{
			int num = docDistributionMap.get(category);
			
			out.write(category + " " + num + "\n");
		}
		
		out.close();
		
		
		//write to build file
		
		System.out.println(buildDistributionMap.size() + " different build distributions");
		
		fstream = new FileWriter("data/buildDistribution.txt");
		
		out = new BufferedWriter(fstream);
	
		out.write("#" + buildDistributionMap.size());
		
		//iteratively push the suffices into file
		categoryList = buildDistributionMap.keySet();
		for (Integer category: categoryList)
		{
			int num = buildDistributionMap.get(category);
			
			out.write(category + " " + num + "\n");
		}
		
		out.close();	
	}
	
	private int classify(Timestamp date)
	{
		Calendar calendar = Calendar.getInstance();
		
		calendar.setTimeInMillis(date.getTime());
		
		int year = calendar.get(Calendar.YEAR);
		int month = calendar.get(Calendar.MONTH);
		
		if (date.after(plotStartDate) && date.before(plotEndDate))
		{
			return (date.getYear() - plotStartDate.getYear()) * 12 
			+ (date.getMonth() - plotStartDate.getMonth());
		}
		
		return -1;
	}
	
	//add current size into the distribution map
	private void addMap(int size, HashMap<Integer, Integer> distributionMap)
	{
		//if this size exists, just increase it by 1
		if (distributionMap.containsKey(size))
		{
			int num = distributionMap.get(size);
			
			//need to remove it first, inefficient :(
			distributionMap.remove(size);
			
			distributionMap.put(size, num + 1);
		}
		//else add it into this map, set the num as 1
		else
		{
			distributionMap.put(size, 1);
		}
	}
	
	
	public static void main(String args[]) throws Exception
	{
		
		String regex = "^\\w+\\.\\w+$";
		
		Pattern pattern = Pattern.compile(regex, Pattern.CASE_INSENSITIVE);
		
		Matcher match = pattern.matcher("xxxx.xxxx");
		
		if (match.find())
		{
			//return words[words.length - 1].toLowerCase();
			
			System.out.println(match.group(0));
		}
		
		
		GetSuffix gs = new GetSuffix(GetSuffix.INSERT_BATCH_SIZE);
		
		//System.out.println(gs.toSuffix("/"));
		
		gs.select(GetSuffix.ROW_NUM_LIMIT);
	}
}